{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.context import SparkContext\n",
    "from pyspark.sql.session import SparkSession\n",
    "from pyspark.mllib.regression import LabeledPoint\n",
    "import numpy as np\n",
    "from pyspark.mllib.regression import LabeledPoint\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 一、项目背景\n",
    "    Build a classifier to categorize webpages as evergreen or non-evergreen\n",
    "    \n",
    "    Stumbleupon是美国的UGC网站，用户分享内容，网站通过用户行为数据构建兴趣图谱和对用户喜好进行一个个性化定位。\n",
    "    Stumbleupon 发布一个比赛，公司提供数据集，包括有标记的训练集和待预测的测试集，根据StumbleUpon提供历史数据，设计分类模型，\n",
    "    预测StumbleUpon提供的网页是否是长期流行，还是短暂流行。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 二、数据集简介\n",
    "    训练集是网页的内容和标记（网页是否是evergreen-长期备受欢迎）\n",
    "    测试集是网页内容\n",
    "    预测目标y：0,1  （0：non-evergreen，1：evergreen）\n",
    "    官网上数据集格式如下："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table><tbody><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>FieldName</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Type</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Description</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>url</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>string</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Url&nbsp;of&nbsp;the&nbsp;webpage&nbsp;to&nbsp;be&nbsp;classified</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>urlid</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>StumbleUpon's&nbsp;unique&nbsp;identifier&nbsp;for&nbsp;each&nbsp;url</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>boilerplate</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>json</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Boilerplate&nbsp;text</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>alchemy_category</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>string</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Alchemy&nbsp;category&nbsp;(per&nbsp;the&nbsp;publicly&nbsp;available&nbsp;Alchemy&nbsp;API&nbsp;found&nbsp;at&nbsp;www.alchemyapi.com)</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>alchemy_category_score</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Alchemy&nbsp;category&nbsp;score&nbsp;(per&nbsp;the&nbsp;publicly&nbsp;available&nbsp;Alchemy&nbsp;API&nbsp;found&nbsp;at&nbsp;www.alchemyapi.com)</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>avglinksize</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Average&nbsp;number&nbsp;of&nbsp;words&nbsp;in&nbsp;each&nbsp;link</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>commonLinkRatio_1</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>#&nbsp;of&nbsp;links&nbsp;sharing&nbsp;at&nbsp;least&nbsp;1&nbsp;word&nbsp;with&nbsp;1&nbsp;other&nbsp;links&nbsp;/&nbsp;#&nbsp;of&nbsp;links</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>commonLinkRatio_2</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>#&nbsp;of&nbsp;links&nbsp;sharing&nbsp;at&nbsp;least&nbsp;1&nbsp;word&nbsp;with&nbsp;2&nbsp;other&nbsp;links&nbsp;/&nbsp;#&nbsp;of&nbsp;links</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>commonLinkRatio_3</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>#&nbsp;of&nbsp;links&nbsp;sharing&nbsp;at&nbsp;least&nbsp;1&nbsp;word&nbsp;with&nbsp;3&nbsp;other&nbsp;links&nbsp;/&nbsp;#&nbsp;of&nbsp;links</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>commonLinkRatio_4</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>#&nbsp;of&nbsp;links&nbsp;sharing&nbsp;at&nbsp;least&nbsp;1&nbsp;word&nbsp;with&nbsp;4&nbsp;other&nbsp;links&nbsp;/&nbsp;#&nbsp;of&nbsp;links</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>compression_ratio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Compression&nbsp;achieved&nbsp;on&nbsp;this&nbsp;page&nbsp;via&nbsp;gzip&nbsp;(measure&nbsp;of&nbsp;redundancy)</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>embed_ratio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Count&nbsp;of&nbsp;number&nbsp;of&nbsp;&lt;embed&gt;&nbsp;&nbsp;usage</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>frameBased</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer&nbsp;(0&nbsp;or&nbsp;1)</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>A&nbsp;page&nbsp;is&nbsp;frame-based&nbsp;(1)&nbsp;if&nbsp;it&nbsp;has&nbsp;no&nbsp;body&nbsp;markup&nbsp;but&nbsp;have&nbsp;a&nbsp;frameset&nbsp;markup</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>frameTagRatio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Ratio&nbsp;of&nbsp;iframe&nbsp;markups&nbsp;over&nbsp;total&nbsp;number&nbsp;of&nbsp;markups</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>hasDomainLink</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer&nbsp;(0&nbsp;or&nbsp;1)</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>True&nbsp;(1)&nbsp;if&nbsp;it&nbsp;contains&nbsp;an&nbsp;&lt;a&gt;&nbsp;&nbsp;with&nbsp;an&nbsp;url&nbsp;with&nbsp;domain</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>html_ratio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Ratio&nbsp;of&nbsp;tags&nbsp;vs&nbsp;text&nbsp;in&nbsp;the&nbsp;page</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>image_ratio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Ratio&nbsp;of&nbsp;&lt;img&gt;&nbsp;tags&nbsp;vs&nbsp;text&nbsp;in&nbsp;the&nbsp;page</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>is_news</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer&nbsp;(0&nbsp;or&nbsp;1)</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>True&nbsp;(1)&nbsp;if&nbsp;StumbleUpon's&nbsp;news&nbsp;classifier&nbsp;determines&nbsp;that&nbsp;this&nbsp;webpage&nbsp;is&nbsp;news</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>lengthyLinkDomain</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer&nbsp;(0&nbsp;or&nbsp;1)</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>True&nbsp;(1)&nbsp;if&nbsp;at&nbsp;least&nbsp;3&nbsp;&lt;a&gt;&nbsp;'s&nbsp;text&nbsp;contains&nbsp;more&nbsp;than&nbsp;30&nbsp;alphanumeric&nbsp;characters</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>linkwordscore</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Percentage&nbsp;of&nbsp;words&nbsp;on&nbsp;the&nbsp;page&nbsp;that&nbsp;are&nbsp;in&nbsp;hyperlink's&nbsp;text</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>news_front_page</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer&nbsp;(0&nbsp;or&nbsp;1)</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>True&nbsp;(1)&nbsp;if&nbsp;StumbleUpon's&nbsp;news&nbsp;classifier&nbsp;determines&nbsp;that&nbsp;this&nbsp;webpage&nbsp;is&nbsp;front-page&nbsp;news</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>non_markup_alphanum_characters</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Page's&nbsp;text's&nbsp;number&nbsp;of&nbsp;alphanumeric&nbsp;characters</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>numberOfLinks</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Number&nbsp;of&nbsp;&lt;a&gt;&nbsp;&nbsp;markups</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>numwords_in_url</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Number&nbsp;of&nbsp;words&nbsp;in&nbsp;url</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>parametrizedLinkRatio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>A&nbsp;link&nbsp;is&nbsp;parametrized&nbsp;if&nbsp;it's&nbsp;url&nbsp;contains&nbsp;parameters&nbsp;&nbsp;or&nbsp;has&nbsp;an&nbsp;attached&nbsp;onClick&nbsp;event</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>spelling_errors_ratio</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>double</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>Ratio&nbsp;of&nbsp;words&nbsp;not&nbsp;found&nbsp;in&nbsp;wiki&nbsp;(considered&nbsp;to&nbsp;be&nbsp;a&nbsp;spelling&nbsp;mistake)</p>\n",
    "</td>\n",
    "</tr><tr><td style=\"background:rgb(240,240,240);\">\n",
    "<p>label</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>integer&nbsp;(0&nbsp;or&nbsp;1)</p>\n",
    "</td>\n",
    "<td style=\"background:rgb(240,240,240);\">\n",
    "<p>User-determined&nbsp;label.&nbsp;Either&nbsp;evergreen&nbsp;(1)&nbsp;or&nbsp;non-evergreen&nbsp;(0);&nbsp;available&nbsp;for&nbsp;train.tsv&nbsp;only</p>\n",
    "</td>\n",
    "</tr></tbody></table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "前四列分别指的是URL、页面的ID、原始的文本内容和分配给页面的类别。接下来22列包含各种各样的数值或者类属特征。最后一列为目标值, 1为长久, 0为短暂。 我们将用简单的方法直接对数值特征做处理。因为每个类属变量是二元的,对这些变量已有一个用1-of-k编码的特征,于是不需要额外提取特征。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 三、加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext(\"local[*]\",\"StumbleUpon\")\n",
    "spark = SparkSession(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始导入数据...\n",
      "共有：7395项数据\n"
     ]
    }
   ],
   "source": [
    "## 读取train.tsv\n",
    "print(\"开始导入数据...\")\n",
    "rawDataWithHeader = sc.textFile(\"data/StumbleUpon/train.tsv\")\n",
    "## 取第一行字段名称数据\n",
    "header = rawDataWithHeader.first()\n",
    "## 剔除字段名（特征名）行，取数据行\n",
    "rawData = rawDataWithHeader.filter(lambda x:x!=header)\n",
    "## 将双引号\"替换为空字符（剔除双引号）\n",
    "rData = rawData.map(lambda x:x.replace(\"\\\"\",\"\"))\n",
    "## 以制表符分割每一行\n",
    "lines = rData.map(lambda x: x.split(\"\\t\"))\n",
    "print(\"共有：\"+str(lines.count())+\"项数据\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html',\n",
       " '4042',\n",
       " '{title:IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries,body:A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.,url:bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html}',\n",
       " 'business',\n",
       " '0.789131',\n",
       " '2.055555556',\n",
       " '0.676470588',\n",
       " '0.205882353',\n",
       " '0.047058824',\n",
       " '0.023529412',\n",
       " '0.443783175',\n",
       " '0',\n",
       " '0',\n",
       " '0.09077381',\n",
       " '0',\n",
       " '0.245831182',\n",
       " '0.003883495',\n",
       " '1',\n",
       " '1',\n",
       " '24',\n",
       " '0',\n",
       " '5424',\n",
       " '170',\n",
       " '8',\n",
       " '0.152941176',\n",
       " '0.079129575',\n",
       " '0']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 四、数据预处理\n",
    "    由于数据格式的问题,我们做一些数据清理的工作,在处理过程中把额外的(\")去掉。数据集中还有一些用\"?\"代替的缺失数据,\n",
    "    本例中,直接用0替换那些缺失数据:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 4.1 处理特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "该数据集的第3个字段是alchemy_category网页分类，是一个离散值特征，要采用OneHotEncode的方式进行编码转换为数值特征，主要过程如下：\n",
    "+ (1) 创建categoriesMap字典，key为网页类别名，value为数字（网页类别名的索引值），每个类别名对应一个索引值\n",
    "+ (2) 根据categoriesMap字典查询每个alchemy_category特征值对应的索引值，例如business的索引值categoryIdx为2\n",
    "+ (3) 根据categoryIdx=2，以OneHotEncodeer的方式转换为一个列表categoryFeatures List，该列表长度为14（统计所有网页类别），categoryIdx=2对应的列表为[0,0,1,0,0,0,0,0,0,0,0,0,0,0]。\n",
    "建立categoriesMap网页分类字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取alchemy_category特征所有类别，并组合成字典，每个类别对应唯一索引\n",
    "categoriesMap = lines.map(lambda row: row[3]).distinct().zipWithIndex().collectAsMap() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "其中，lines.map()表示处理之前读取的数据的每一行，.map(lambda row: row[0])表示读取第0个字段，.distinct()保留不重复数据，.zipWithIndex()将第3个字段中不重复的数据进行编号，.collectAsMap()转换为dict字典格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'business': 0,\n",
       " 'sports': 1,\n",
       " '?': 2,\n",
       " 'arts_entertainment': 3,\n",
       " 'gaming': 4,\n",
       " 'culture_politics': 5,\n",
       " 'computer_internet': 6,\n",
       " 'law_crime': 7,\n",
       " 'religion': 8,\n",
       " 'weather': 9,\n",
       " 'unknown': 10,\n",
       " 'recreation': 11,\n",
       " 'health': 12,\n",
       " 'science_technology': 13}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看 categoriesMap 字典\n",
    "categoriesMap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对于第4~25字段的数值特征，要转换为数值，用float函数将字符串转换为数值，同时简单处理缺失值”?”为0.\n",
    "整个处理特征的过程可以封装成一个函数："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(v):\n",
    "    \"\"\"\n",
    "    处理数值特征的转换函数\n",
    "    在所有数值类型特征中，有些值为 “？”\n",
    "    我们将“？”直接替换为0，\n",
    "    如果有值，则转化为float类型\n",
    "    \"\"\"\n",
    "    return (0 if v==\"?\" else float(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_features(line, categoriesMap, featureEnd):\n",
    "    \"\"\"\n",
    "        处理特征，line为字段行，\n",
    "        categoriesMap为网页分类字典，\n",
    "        featureEnd为特征结束位置，此例为25\n",
    "    \"\"\"\n",
    "    ## 处理alchemy_category网页分类特征\n",
    "    categoryIdx = categoriesMap[line[3]]\n",
    "    OneHot = np.zeros(len(categoriesMap))\n",
    "    OneHot[categoryIdx] = 1\n",
    "    ## 处理数值特征\n",
    "    numericalFeatures = [convert(value) for value in line[4:featureEnd]]\n",
    "    # 返回拼接的总特征列表\n",
    "    return np.concatenate((OneHot, numericalFeatures))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.2 处理label分类标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_label(line):\n",
    "    return float(line[-1])  # 最后一个字段为类别标签"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.3 构建模型所需数据格式\n",
    "Spark Mllib分类任务支持的数据类型为LabeledPoint格式，LabeledPoint数据由标签label和特征feature组成。构建LabeledPoint数据："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "数据预处理，此行代码包含以下操作\n",
    "1、特征变量与目标变量拆分\n",
    "2、alchemy_category类别变量OneHot编码\n",
    "3、数值型特征由str类型转为float类型\n",
    "\n",
    "\"\"\"\n",
    "labelpointRDD = lines.map(lambda r: LabeledPoint(process_label(r),\n",
    "                                                 process_features(r,categoriesMap, len(r)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(0.0, [1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.789131,2.055555556,0.676470588,0.205882353,0.047058824,0.023529412,0.443783175,0.0,0.0,0.09077381,0.0,0.245831182,0.003883495,1.0,1.0,24.0,0.0,5424.0,170.0,8.0,0.152941176,0.079129575])]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看一个训练样本\n",
    "labelpointRDD.take(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.4 划分训练集、验证集以及测试集\n",
    "按照7:1:2的比例划分训练集、验证集以及测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集样本个数：5141\n",
      "验证集样本个数：786\n",
      "测试集样本个数： 1468\n"
     ]
    }
   ],
   "source": [
    "## 划分训练集、验证集和测试集\n",
    "(trainData, validationData, testData) = labelpointRDD.randomSplit([7,1,2])\n",
    "print(\"训练集样本个数：\"+str(trainData.count()))\n",
    "print(\"验证集样本个数：\"+str(validationData.count()))\n",
    "print(\"测试集样本个数：\",str(testData.count()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[23] at RDD at PythonRDD.scala:53"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将数据暂存在内存中，加快后续运算效率\n",
    "trainData.persist()\n",
    "validationData.persist()\n",
    "testData.persist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 五、训练模型\n",
    "选择Spark MLlib中的决策树DecisionTree模块中的trainClassifier方法进行训练并建立模型：\n",
    "\n",
    "DecisionTree.trainClassifier(input, numClasses, categoricalFeaturesInfo, impurity,maxDepth,maxBins)\n",
    "参数说明如下：\n",
    "\n",
    "+ (1) input：输入的训练数据，数据格式为LabeledPoint数据\n",
    "+ (2) numClasses：指定分类数目\n",
    "+ (3) categoricalFeaturesInfo：设置分类特征字段信息，本例采用OneHot编码处理分类特征字段，故这里设置为空字典dict()\n",
    "+ (4) impurity：决策树的impurity评估方法（划分的度量选择）：gini基尼系数，entropy熵\n",
    "+ (5) maxDepth：决策树最大深度\n",
    "+ (6) maxBins：决策树每个节点的最大分支数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.mllib.tree import DecisionTree\n",
    "model = DecisionTree.trainClassifier(trainData, numClasses=2,categoricalFeaturesInfo={}, impurity=\"entropy\", maxDepth=5,maxBins=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 六、模型评估\n",
    "使用AUC(Area under the Curve of ROC)来对模型进行评估，接收者操作特征(Receiver Operating Characteristic , ROC)曲线是一种比较分类器模型有用的可视化工具。\n",
    "\n",
    "ROC曲线显示了给定模型的真正例率(TPR=TP/P)(纵轴)和假正例率(FPR=FP/N)(横轴)之间的权衡。TPR的增加以FPR的增加为代价。ROC曲线下方的面积是模型准确率的度量：AUC\n",
    "\n",
    "AUC=1：预测准确率100%\n",
    "0.5 < AUC <1：优于随机猜测，具有预测意义\n",
    "AUC=0.5: 与随机猜测一样，没有预测意义\n",
    "AUC<0.5: 比随机预测还差\n",
    "Spark Mllib提供了BinaryClassificationMetrics计算AUC的方法。\n",
    "\n",
    "首先创建predict_real列表，列表的每个元素为一个元组(predict,real)，其中predict为预测结果，real为实际标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1.0, 0.0), (1.0, 1.0), (0.0, 0.0), (0.0, 0.0), (0.0, 0.0)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 创建predict_real列表\n",
    "predict = model.predict(validationData.map(lambda p:p.features))\n",
    "predict_real = predict.zip(validationData.map(lambda p: p.label))\n",
    "predict_real.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC=0.6412509307520476\n"
     ]
    }
   ],
   "source": [
    "# 接着使用BinaryClassificationMetrics计算AUC\n",
    "from pyspark.mllib.evaluation import BinaryClassificationMetrics\n",
    "metrics = BinaryClassificationMetrics(predict_real)\n",
    "print(\"AUC=\"+str(metrics.areaUnderROC))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 七、使用模型进行预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始导入数据...\n",
      "网址：http://www.lynnskitchenadventures.com/2009/04/homemade-enchilada-sauce.html\n",
      " ===>预测结果为: 1.0说明: 长久型(evergreen)网页\n",
      "\n",
      "网址：http://lolpics.se/18552-stun-grenade-ar\n",
      " ===>预测结果为: 0.0说明: 暂时型(ephemeral)网页\n",
      "\n",
      "网址：http://www.xcelerationfitness.com/treadmills.html\n",
      " ===>预测结果为: 0.0说明: 暂时型(ephemeral)网页\n",
      "\n",
      "网址：http://www.bloomberg.com/news/2012-02-06/syria-s-assad-deploys-tactics-of-father-to-crush-revolt-threatening-reign.html\n",
      " ===>预测结果为: 0.0说明: 暂时型(ephemeral)网页\n",
      "\n",
      "网址：http://www.wired.com/gadgetlab/2011/12/stem-turns-lemons-and-limes-into-juicy-atomizers/\n",
      " ===>预测结果为: 0.0说明: 暂时型(ephemeral)网页\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## 使用模型进行预测\n",
    "def predictData(sc,model,categoriesMap):\n",
    "    print(\"开始导入数据...\")\n",
    "    rawDataWithHeader = sc.textFile(\"data/StumbleUpon/test.tsv\")\n",
    "    ## 取第一项数据\n",
    "    header = rawDataWithHeader.first()\n",
    "    ## 剔除字段名（特征名）行，取数据行\n",
    "    rawData = rawDataWithHeader.filter(lambda x:x!=header)\n",
    "    ## 将双引号\"替换为空字符（剔除双引号）\n",
    "    rData = rawData.map(lambda x:x.replace(\"\\\"\",\"\"))\n",
    "    ## 以制表符分割每一行\n",
    "    lines = rData.map(lambda x: x.split(\"\\t\"))\n",
    "    ## 预处理测试数据集\n",
    "    testDataRDD=lines.map(lambda r: (r[0], process_features(r, categoriesMap, len(r))))\n",
    "    DescDict={0:\"暂时型(ephemeral)网页\",\n",
    "              1:\"长久型(evergreen)网页\"}\n",
    "    ## 预测前5项数据\n",
    "    for testData in testDataRDD.take(5):\n",
    "        predictResult=model.predict(testData[1])\n",
    "        print(\"网址：\"+str(testData[0])+\"\\n\"+\" ===>预测结果为: \"+str(predictResult) + \"说明: \"+DescDict[predictResult]+\"\\n\")\n",
    "\n",
    "predictData(sc,model,categoriesMap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
